{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Let's Create Our Credit Card Dataset\n",
    "- There two main font variations used in credit cards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "\n",
    "cc1 = cv2.imread('creditcard_digits1.jpg', 0)\n",
    "cv2.imshow(\"Digits 1\", cc1)\n",
    "cv2.waitKey(0)\n",
    "cc2 = cv2.imread('creditcard_digits2.jpg', 0)\n",
    "cv2.imshow(\"Digits 2\", cc2)\n",
    "cv2.waitKey(0)\n",
    "cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "cc1 = cv2.imread('creditcard_digits2.jpg', 0)\n",
    "_, th2 = cv2.threshold(cc1, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)\n",
    "cv2.imshow(\"Digits 2 Thresholded\", th2)\n",
    "cv2.waitKey(0)\n",
    "    \n",
    "cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Now let's get generate an Augumentated Dataset from these two samples \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./credit_card/train/0\n",
      "./credit_card/train/1\n",
      "./credit_card/train/2\n",
      "./credit_card/train/3\n",
      "./credit_card/train/4\n",
      "./credit_card/train/5\n",
      "./credit_card/train/6\n",
      "./credit_card/train/7\n",
      "./credit_card/train/8\n",
      "./credit_card/train/9\n",
      "./credit_card/test/0\n",
      "./credit_card/test/1\n",
      "./credit_card/test/2\n",
      "./credit_card/test/3\n",
      "./credit_card/test/4\n",
      "./credit_card/test/5\n",
      "./credit_card/test/6\n",
      "./credit_card/test/7\n",
      "./credit_card/test/8\n",
      "./credit_card/test/9\n"
     ]
    }
   ],
   "source": [
    "#Create our dataset directories\n",
    "\n",
    "import os\n",
    "\n",
    "def makedir(directory):\n",
    "    \"\"\"Creates a new directory if it does not exist\"\"\"\n",
    "    if not os.path.exists(directory):\n",
    "        os.makedirs(directory)\n",
    "        return None, 0\n",
    "    \n",
    "for i in range(0,10):\n",
    "    directory_name = \"./credit_card/train/\"+str(i)\n",
    "    print(directory_name)\n",
    "    makedir(directory_name) \n",
    "\n",
    "for i in range(0,10):\n",
    "    directory_name = \"./credit_card/test/\"+str(i)\n",
    "    print(directory_name)\n",
    "    makedir(directory_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Let's make our Data Augmentation Functions\n",
    "These are used to perform image manipulation and pre-processing tasks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import numpy as np \n",
    "import random\n",
    "import cv2\n",
    "from scipy.ndimage import convolve\n",
    "\n",
    "def DigitAugmentation(frame, dim = 32):\n",
    "    \"\"\"Randomly alters the image using noise, pixelation and streching image functions\"\"\"\n",
    "    frame = cv2.resize(frame, None, fx=2, fy=2, interpolation = cv2.INTER_CUBIC)\n",
    "    frame = cv2.cvtColor(frame, cv2.COLOR_GRAY2RGB)\n",
    "    random_num = np.random.randint(0,9)\n",
    "\n",
    "    if (random_num % 2 == 0):\n",
    "        frame = add_noise(frame)\n",
    "    if(random_num % 3 == 0):\n",
    "        frame = pixelate(frame)\n",
    "    if(random_num % 2 == 0):\n",
    "        frame = stretch(frame)\n",
    "    frame = cv2.resize(frame, (dim, dim), interpolation = cv2.INTER_AREA)\n",
    "\n",
    "    return frame \n",
    "\n",
    "def add_noise(image):\n",
    "    \"\"\"Addings noise to image\"\"\"\n",
    "    prob = random.uniform(0.01, 0.05)\n",
    "    rnd = np.random.rand(image.shape[0], image.shape[1])\n",
    "    noisy = image.copy()\n",
    "    noisy[rnd < prob] = 0\n",
    "    noisy[rnd > 1 - prob] = 1\n",
    "    return noisy\n",
    "\n",
    "def pixelate(image):\n",
    "    \"Pixelates an image by reducing the resolution then upscaling it\"\n",
    "    dim = np.random.randint(8,12)\n",
    "    image = cv2.resize(image, (dim, dim), interpolation = cv2.INTER_AREA)\n",
    "    image = cv2.resize(image, (16, 16), interpolation = cv2.INTER_AREA)\n",
    "    return image\n",
    "\n",
    "def stretch(image):\n",
    "    \"Randomly applies different degrees of stretch to image\"\n",
    "    ran = np.random.randint(0,3)*2\n",
    "    if np.random.randint(0,2) == 0:\n",
    "        frame = cv2.resize(image, (32, ran+32), interpolation = cv2.INTER_AREA)\n",
    "        return frame[int(ran/2):int(ran+32)-int(ran/2), 0:32]\n",
    "    else:\n",
    "        frame = cv2.resize(image, (ran+32, 32), interpolation = cv2.INTER_AREA)\n",
    "        return frame[0:32, int(ran/2):int(ran+32)-int(ran/2)]\n",
    "    \n",
    "def pre_process(image, inv = False):\n",
    "    \"\"\"Uses OTSU binarization on an image\"\"\"\n",
    "    try:\n",
    "        gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n",
    "    except:\n",
    "        gray_image = image\n",
    "        pass\n",
    "    \n",
    "    if inv == False:\n",
    "        _, th2 = cv2.threshold(gray_image, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)\n",
    "    else:\n",
    "        _, th2 = cv2.threshold(gray_image, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)\n",
    "    resized = cv2.resize(th2, (32,32), interpolation = cv2.INTER_AREA)\n",
    "    return resized"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing our augmentation functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "cc1 = cv2.imread('creditcard_digits2.jpg', 0)\n",
    "_, th2 = cv2.threshold(cc1, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)\n",
    "cv2.imshow(\"cc1\", th2)\n",
    "cv2.waitKey(0)\n",
    "cv2.destroyAllWindows()\n",
    "\n",
    "# This is the coordinates of the region enclosing  the first digit\n",
    "# This is preset and was done manually based on this specific image\n",
    "region = [(0, 0), (35, 48)]\n",
    "\n",
    "# Assigns values to each region for ease of interpretation\n",
    "top_left_y = region[0][1]\n",
    "bottom_right_y = region[1][1]\n",
    "top_left_x = region[0][0]\n",
    "bottom_right_x = region[1][0]\n",
    "\n",
    "for i in range(0,1): #We only look at the first digit in testing out augmentation functions\n",
    "    roi = cc1[top_left_y:bottom_right_y, top_left_x:bottom_right_x]\n",
    "    for j in range(0,10):\n",
    "        roi2 = DigitAugmentation(roi)\n",
    "        roi_otsu = pre_process(roi2, inv = False)\n",
    "        cv2.imshow(\"otsu\", roi_otsu)\n",
    "        cv2.waitKey(0)\n",
    "        \n",
    "cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating our Training Data (1000 variations of each font type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating 2000 Images for each digit in creditcard_digits1 - TRAINING DATA\n",
    "\n",
    "# Load our first image\n",
    "cc1 = cv2.imread('creditcard_digits1.jpg', 0)\n",
    "\n",
    "_, th2 = cv2.threshold(cc1, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)\n",
    "cv2.imshow(\"cc1\", th2)\n",
    "cv2.imshow(\"creditcard_digits1\", cc1)\n",
    "cv2.waitKey(0)\n",
    "cv2.destroyAllWindows()\n",
    "\n",
    "region = [(2, 19), (50, 72)]\n",
    "\n",
    "top_left_y = region[0][1]\n",
    "bottom_right_y = region[1][1]\n",
    "top_left_x = region[0][0]\n",
    "bottom_right_x = region[1][0]\n",
    "\n",
    "for i in range(0,10):   \n",
    "    # We jump the next digit each time we loop\n",
    "    if i > 0:\n",
    "        top_left_x = top_left_x + 59\n",
    "        bottom_right_x = bottom_right_x + 59\n",
    "\n",
    "    roi = cc1[top_left_y:bottom_right_y, top_left_x:bottom_right_x]\n",
    "    print(\"Augmenting Digit - \", str(i))\n",
    "    # We create 200 versions of each image for our dataset\n",
    "    for j in range(0,2000):\n",
    "        roi2 = DigitAugmentation(roi)\n",
    "        roi_otsu = pre_process(roi2, inv = True)\n",
    "        cv2.imwrite(\"./credit_card/train/\"+str(i)+\"./_1_\"+str(j)+\".jpg\", roi_otsu)\n",
    "cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating 2000 Images for each digit in creditcard_digits2 - TRAINING DATA\n",
    "\n",
    "cc1 = cv2.imread('creditcard_digits2.jpg', 0)\n",
    "_, th2 = cv2.threshold(cc1, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)\n",
    "cv2.imshow(\"cc1\", th2)\n",
    "cv2.waitKey(0)\n",
    "cv2.destroyAllWindows()\n",
    "\n",
    "region = [(0, 0), (35, 48)]\n",
    "\n",
    "top_left_y = region[0][1]\n",
    "bottom_right_y = region[1][1]\n",
    "top_left_x = region[0][0]\n",
    "bottom_right_x = region[1][0]\n",
    "\n",
    "for i in range(0,10):   \n",
    "    if i > 0:\n",
    "        # We jump the next digit each time we loop\n",
    "        top_left_x = top_left_x + 35\n",
    "        bottom_right_x = bottom_right_x + 35\n",
    "\n",
    "    roi = cc1[top_left_y:bottom_right_y, top_left_x:bottom_right_x]\n",
    "    print(\"Augmenting Digit - \", str(i))\n",
    "    # We create 200 versions of each image for our dataset\n",
    "    for j in range(0,2000):\n",
    "        roi2 = DigitAugmentation(roi)\n",
    "        roi_otsu = pre_process(roi2, inv = False)\n",
    "        cv2.imwrite(\"./credit_card/train/\"+str(i)+\"./_2_\"+str(j)+\".jpg\", roi_otsu)\n",
    "        cv2.imshow(\"otsu\", roi_otsu)\n",
    "        print(\"-\")\n",
    "        cv2.waitKey(0)\n",
    "cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating 200 Images for each digit in creditcard_digits1 - TEST DATA\n",
    "\n",
    "# Load our first image\n",
    "cc1 = cv2.imread('creditcard_digits1.jpg', 0)\n",
    "\n",
    "_, th2 = cv2.threshold(cc1, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)\n",
    "cv2.imshow(\"cc1\", th2)\n",
    "cv2.imshow(\"creditcard_digits1\", cc1)\n",
    "cv2.waitKey(0)\n",
    "cv2.destroyAllWindows()\n",
    "\n",
    "region = [(2, 19), (50, 72)]\n",
    "\n",
    "top_left_y = region[0][1]\n",
    "bottom_right_y = region[1][1]\n",
    "top_left_x = region[0][0]\n",
    "bottom_right_x = region[1][0]\n",
    "\n",
    "for i in range(0,10):   \n",
    "    # We jump the next digit each time we loop\n",
    "    if i > 0:\n",
    "        top_left_x = top_left_x + 59\n",
    "        bottom_right_x = bottom_right_x + 59\n",
    "\n",
    "    roi = cc1[top_left_y:bottom_right_y, top_left_x:bottom_right_x]\n",
    "    print(\"Augmenting Digit - \", str(i))\n",
    "    # We create 200 versions of each image for our dataset\n",
    "    for j in range(0,2000):\n",
    "        roi2 = DigitAugmentation(roi)\n",
    "        roi_otsu = pre_process(roi2, inv = True)\n",
    "        cv2.imwrite(\"./credit_card/test/\"+str(i)+\"./_1_\"+str(j)+\".jpg\", roi_otsu)\n",
    "cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating 200 Images for each digit in creditcard_digits2 - TEST DATA\n",
    "\n",
    "cc1 = cv2.imread('creditcard_digits2.jpg', 0)\n",
    "_, th2 = cv2.threshold(cc1, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)\n",
    "cv2.imshow(\"cc1\", th2)\n",
    "cv2.waitKey(0)\n",
    "cv2.destroyAllWindows()\n",
    "\n",
    "region = [(0, 0), (35, 48)]\n",
    "\n",
    "top_left_y = region[0][1]\n",
    "bottom_right_y = region[1][1]\n",
    "top_left_x = region[0][0]\n",
    "bottom_right_x = region[1][0]\n",
    "\n",
    "for i in range(0,10):   \n",
    "    if i > 0:\n",
    "        # We jump the next digit each time we loop\n",
    "        top_left_x = top_left_x + 35\n",
    "        bottom_right_x = bottom_right_x + 35\n",
    "\n",
    "    roi = cc1[top_left_y:bottom_right_y, top_left_x:bottom_right_x]\n",
    "    print(\"Augmenting Digit - \", str(i))\n",
    "    # We create 200 versions of each image for our dataset\n",
    "    for j in range(0,2000):\n",
    "        roi2 = DigitAugmentation(roi)\n",
    "        roi_otsu = pre_process(roi2, inv = False)\n",
    "        cv2.imwrite(\"./credit_card/test/\"+str(i)+\"./_2_\"+str(j)+\".jpg\", roi_otsu)\n",
    "        cv2.imshow(\"otsu\", roi_otsu)\n",
    "        print(\"-\")\n",
    "        cv2.waitKey(0)\n",
    "cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Creating our Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Activation, Dropout, Flatten, Dense\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.layers import Conv2D, MaxPooling2D, ZeroPadding2D\n",
    "from keras import optimizers\n",
    "import keras\n",
    "\n",
    "input_shape = (32, 32, 3)\n",
    "img_width = 32\n",
    "img_height = 32\n",
    "num_classes = 10\n",
    "nb_train_samples = 10000\n",
    "nb_validation_samples = 2000\n",
    "batch_size = 16\n",
    "epochs = 1\n",
    "\n",
    "train_data_dir = './credit_card/train'\n",
    "validation_data_dir = './credit_card/test'\n",
    "\n",
    "# Creating our data generator for our test data\n",
    "validation_datagen = ImageDataGenerator(\n",
    "    # used to rescale the pixel values from [0, 255] to [0, 1] interval\n",
    "    rescale = 1./255)\n",
    "\n",
    "# Creating our data generator for our training data\n",
    "train_datagen = ImageDataGenerator(\n",
    "      rescale = 1./255,              # normalize pixel values to [0,1]\n",
    "      rotation_range = 10,           # randomly applies rotations\n",
    "      width_shift_range = 0.25,       # randomly applies width shifting\n",
    "      height_shift_range = 0.25,      # randomly applies height shifting\n",
    "      shear_range=0.5,\n",
    "      zoom_range=0.5,\n",
    "      horizontal_flip = False,        # randonly flips the image\n",
    "      fill_mode = 'nearest')         # uses the fill mode nearest to fill gaps created by the above\n",
    "\n",
    "# Specify criteria about our training data, such as the directory, image size, batch size and type \n",
    "# automagically retrieve images and their classes for train and validation sets\n",
    "train_generator = train_datagen.flow_from_directory(\n",
    "        train_data_dir,\n",
    "        target_size = (img_width, img_height),\n",
    "        batch_size = batch_size,\n",
    "        class_mode = 'categorical')\n",
    "\n",
    "validation_generator = validation_datagen.flow_from_directory(\n",
    "        validation_data_dir,\n",
    "        target_size = (img_width, img_height),\n",
    "        batch_size = batch_size,\n",
    "        class_mode = 'categorical',\n",
    "        shuffle = False)    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating out Model based on the LeNet CNN Architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create model\n",
    "model = Sequential()\n",
    "\n",
    "# 2 sets of CRP (Convolution, RELU, Pooling)\n",
    "model.add(Conv2D(20, (5, 5),\n",
    "                 padding = \"same\", \n",
    "                 input_shape = input_shape))\n",
    "model.add(Activation(\"relu\"))\n",
    "model.add(MaxPooling2D(pool_size = (2, 2), strides = (2, 2)))\n",
    "\n",
    "model.add(Conv2D(50, (5, 5),\n",
    "                 padding = \"same\"))\n",
    "model.add(Activation(\"relu\"))\n",
    "model.add(MaxPooling2D(pool_size = (2, 2), strides = (2, 2)))\n",
    "\n",
    "# Fully connected layers (w/ RELU)\n",
    "model.add(Flatten())\n",
    "model.add(Dense(500))\n",
    "model.add(Activation(\"relu\"))\n",
    "\n",
    "# Softmax (for classification)\n",
    "model.add(Dense(num_classes))\n",
    "model.add(Activation(\"softmax\"))\n",
    "           \n",
    "model.compile(loss = 'categorical_crossentropy',\n",
    "              optimizer = keras.optimizers.Adadelta(),\n",
    "              metrics = ['accuracy'])\n",
    "    \n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training our Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.optimizers import RMSprop\n",
    "from keras.callbacks import ModelCheckpoint, EarlyStopping\n",
    "                   \n",
    "checkpoint = ModelCheckpoint(\"/home/deeplearningcv/DeepLearningCV/Trained Models/creditcard.h5\",\n",
    "                             monitor=\"val_loss\",\n",
    "                             mode=\"min\",\n",
    "                             save_best_only = True,\n",
    "                             verbose=1)\n",
    "\n",
    "earlystop = EarlyStopping(monitor = 'val_loss', \n",
    "                          min_delta = 0, \n",
    "                          patience = 3,\n",
    "                          verbose = 1,\n",
    "                          restore_best_weights = True)\n",
    "\n",
    "# we put our call backs into a callback list\n",
    "callbacks = [earlystop, checkpoint]\n",
    "\n",
    "# Note we use a very small learning rate \n",
    "model.compile(loss = 'categorical_crossentropy',\n",
    "              optimizer = RMSprop(lr = 0.001),\n",
    "              metrics = ['accuracy'])\n",
    "\n",
    "nb_train_samples = 20000\n",
    "nb_validation_samples = 4000\n",
    "epochs = 5\n",
    "batch_size = 16\n",
    "\n",
    "history = model.fit_generator(\n",
    "    train_generator,\n",
    "    steps_per_epoch = nb_train_samples // batch_size,\n",
    "    epochs = epochs,\n",
    "    callbacks = callbacks,\n",
    "    validation_data = validation_generator,\n",
    "    validation_steps = nb_validation_samples // batch_size)\n",
    "\n",
    "model.save(\"/home/deeplearningcv/DeepLearningCV/Trained Models/creditcard.h5\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Extract a Credit Card from the backgroud\n",
    "#### NOTE:\n",
    "You may need to install imutils \n",
    "run *pip install imutils* in terminal and restart your kernal to install"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import numpy as np\n",
    "import imutils\n",
    "from skimage.filters import threshold_adaptive\n",
    "import os\n",
    "\n",
    "def order_points(pts):\n",
    "    # initialzie a list of coordinates that will be ordered\n",
    "    # such that the first entry in the list is the top-left,\n",
    "    # the second entry is the top-right, the third is the\n",
    "    # bottom-right, and the fourth is the bottom-left\n",
    "    rect = np.zeros((4, 2), dtype = \"float32\")\n",
    "\n",
    "    # the top-left point will have the smallest sum, whereas\n",
    "    # the bottom-right point will have the largest sum\n",
    "    s = pts.sum(axis = 1)\n",
    "    rect[0] = pts[np.argmin(s)]\n",
    "    rect[2] = pts[np.argmax(s)]\n",
    "\n",
    "    # now, compute the difference between the points, the\n",
    "    # top-right point will have the smallest difference,\n",
    "    # whereas the bottom-left will have the largest difference\n",
    "    diff = np.diff(pts, axis = 1)\n",
    "    rect[1] = pts[np.argmin(diff)]\n",
    "    rect[3] = pts[np.argmax(diff)]\n",
    "\n",
    "    # return the ordered coordinates\n",
    "    return rect\n",
    "\n",
    "def four_point_transform(image, pts):\n",
    "    # obtain a consistent order of the points and unpack them\n",
    "    # individually\n",
    "    rect = order_points(pts)\n",
    "    (tl, tr, br, bl) = rect\n",
    "\n",
    "    # compute the width of the new image, which will be the\n",
    "    # maximum distance between bottom-right and bottom-left\n",
    "    # x-coordiates or the top-right and top-left x-coordinates\n",
    "    widthA = np.sqrt(((br[0] - bl[0]) ** 2) + ((br[1] - bl[1]) ** 2))\n",
    "    widthB = np.sqrt(((tr[0] - tl[0]) ** 2) + ((tr[1] - tl[1]) ** 2))\n",
    "    maxWidth = max(int(widthA), int(widthB))\n",
    "\n",
    "    # compute the height of the new image, which will be the\n",
    "    # maximum distance between the top-right and bottom-right\n",
    "    # y-coordinates or the top-left and bottom-left y-coordinates\n",
    "    heightA = np.sqrt(((tr[0] - br[0]) ** 2) + ((tr[1] - br[1]) ** 2))\n",
    "    heightB = np.sqrt(((tl[0] - bl[0]) ** 2) + ((tl[1] - bl[1]) ** 2))\n",
    "    maxHeight = max(int(heightA), int(heightB))\n",
    "\n",
    "    # now that we have the dimensions of the new image, construct\n",
    "    # the set of destination points to obtain a \"birds eye view\",\n",
    "    # (i.e. top-down view) of the image, again specifying points\n",
    "    # in the top-left, top-right, bottom-right, and bottom-left\n",
    "    # order\n",
    "    dst = np.array([\n",
    "        [0, 0],\n",
    "        [maxWidth - 1, 0],\n",
    "        [maxWidth - 1, maxHeight - 1],\n",
    "        [0, maxHeight - 1]], dtype = \"float32\")\n",
    "\n",
    "    # compute the perspective transform matrix and then apply it\n",
    "    M = cv2.getPerspectiveTransform(rect, dst)\n",
    "    warped = cv2.warpPerspective(image, M, (maxWidth, maxHeight))\n",
    "\n",
    "    # return the warped image\n",
    "    return warped\n",
    "\n",
    "def doc_Scan(image):\n",
    "    orig_height, orig_width = image.shape[:2]\n",
    "    ratio = image.shape[0] / 500.0\n",
    "\n",
    "    orig = image.copy()\n",
    "    image = imutils.resize(image, height = 500)\n",
    "    orig_height, orig_width = image.shape[:2]\n",
    "    Original_Area = orig_height * orig_width\n",
    "    \n",
    "    # convert the image to grayscale, blur it, and find edges\n",
    "    # in the image\n",
    "    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n",
    "    gray = cv2.GaussianBlur(gray, (5, 5), 0)\n",
    "    edged = cv2.Canny(gray, 75, 200)\n",
    "\n",
    "    cv2.imshow(\"Image\", image)\n",
    "    cv2.imshow(\"Edged\", edged)\n",
    "    cv2.waitKey(0)\n",
    "    # show the original image and the edge detected image\n",
    "\n",
    "    # find the contours in the edged image, keeping only the\n",
    "    # largest ones, and initialize the screen contour\n",
    "    _, contours, hierarchy  = cv2.findContours(edged.copy(), cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)\n",
    "    contours = sorted(contours, key = cv2.contourArea, reverse = True)[:5]\n",
    "    \n",
    "    # loop over the contours\n",
    "    for c in contours:\n",
    "\n",
    "        # approximate the contour\n",
    "        area = cv2.contourArea(c)\n",
    "        if area < (Original_Area/3):\n",
    "            print(\"Error Image Invalid\")\n",
    "            return(\"ERROR\")\n",
    "        peri = cv2.arcLength(c, True)\n",
    "        approx = cv2.approxPolyDP(c, 0.02 * peri, True)\n",
    "\n",
    "        # if our approximated contour has four points, then we\n",
    "        # can assume that we have found our screen\n",
    "        if len(approx) == 4:\n",
    "            screenCnt = approx\n",
    "            break\n",
    "\n",
    "    # show the contour (outline) of the piece of paper\n",
    "    cv2.drawContours(image, [screenCnt], -1, (0, 255, 0), 2)\n",
    "    cv2.imshow(\"Outline\", image)\n",
    "\n",
    "    warped = four_point_transform(orig, screenCnt.reshape(4, 2) * ratio)\n",
    "    # convert the warped image to grayscale, then threshold it\n",
    "    # to give it that 'black and white' paper effect\n",
    "    cv2.resize(warped, (640,403), interpolation = cv2.INTER_AREA)\n",
    "    cv2.imwrite(\"credit_card_color.jpg\", warped)\n",
    "    warped = cv2.cvtColor(warped, cv2.COLOR_BGR2GRAY)\n",
    "    warped = warped.astype(\"uint8\") * 255\n",
    "    cv2.imshow(\"Extracted Credit Card\", warped)\n",
    "    cv2.waitKey(0)\n",
    "    cv2.destroyAllWindows()\n",
    "    return warped"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract our Credit Card and the Region of Interest (ROI)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "image = cv2.imread('test_card.jpg')\n",
    "image = doc_Scan(image)\n",
    "\n",
    "region = [(55, 210), (640, 290)]\n",
    "\n",
    "top_left_y = region[0][1]\n",
    "bottom_right_y = region[1][1]\n",
    "top_left_x = region[0][0]\n",
    "bottom_right_x = region[1][0]\n",
    "\n",
    "# Extracting the area were the credit numbers are located\n",
    "roi = image[top_left_y:bottom_right_y, top_left_x:bottom_right_x]\n",
    "cv2.imshow(\"Region\", roi)\n",
    "cv2.imwrite(\"credit_card_extracted_digits.jpg\", roi)\n",
    "cv2.waitKey(0)\n",
    "cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading our trained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import load_model\n",
    "import keras\n",
    "\n",
    "classifier = load_model('/home/deeplearningcv/DeepLearningCV/Trained Models/creditcard.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Let's test on our extracted image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5\n",
      "3\n",
      "5\n",
      "5\n",
      "2\n",
      "2\n",
      "0\n",
      "3\n",
      "2\n",
      "3\n",
      "9\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "def x_cord_contour(contours):\n",
    "    #Returns the X cordinate for the contour centroid\n",
    "    if cv2.contourArea(contours) > 10:\n",
    "        M = cv2.moments(contours)\n",
    "        return (int(M['m10']/M['m00']))\n",
    "    else:\n",
    "        pass\n",
    "\n",
    "img = cv2.imread('credit_card_extracted_digits.jpg')\n",
    "orig_img = cv2.imread('credit_card_color.jpg')\n",
    "gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)\n",
    "cv2.imshow(\"image\", img)\n",
    "cv2.waitKey(0)\n",
    "\n",
    "# Blur image then find edges using Canny \n",
    "blurred = cv2.GaussianBlur(gray, (5, 5), 0)\n",
    "#cv2.imshow(\"blurred\", blurred)\n",
    "#cv2.waitKey(0)\n",
    "\n",
    "edged = cv2.Canny(blurred, 30, 150)\n",
    "#cv2.imshow(\"edged\", edged)\n",
    "#cv2.waitKey(0)\n",
    "\n",
    "# Find Contours\n",
    "_, contours, _ = cv2.findContours(edged.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n",
    "\n",
    "#Sort out contours left to right by using their x cordinates\n",
    "contours = sorted(contours, key=cv2.contourArea, reverse=True)[:13] #Change this to 16 to get all digits\n",
    "contours = sorted(contours, key = x_cord_contour, reverse = False)\n",
    "\n",
    "# Create empty array to store entire number\n",
    "full_number = []\n",
    "\n",
    "# loop over the contours\n",
    "for c in contours:\n",
    "    # compute the bounding box for the rectangle\n",
    "    (x, y, w, h) = cv2.boundingRect(c)    \n",
    "    if w >= 5 and h >= 25 and cv2.contourArea(c) < 1000:\n",
    "        roi = blurred[y:y + h, x:x + w]\n",
    "        #ret, roi = cv2.threshold(roi, 20, 255,cv2.THRESH_BINARY_INV)\n",
    "        cv2.imshow(\"ROI1\", roi)\n",
    "        roi_otsu = pre_process(roi, True)\n",
    "        cv2.imshow(\"ROI2\", roi_otsu)\n",
    "        roi_otsu = cv2.cvtColor(roi_otsu, cv2.COLOR_GRAY2RGB)\n",
    "        roi_otsu = keras.preprocessing.image.img_to_array(roi_otsu)\n",
    "        roi_otsu = roi_otsu * 1./255\n",
    "        roi_otsu = np.expand_dims(roi_otsu, axis=0)\n",
    "        image = np.vstack([roi_otsu])\n",
    "        label = str(classifier.predict_classes(image, batch_size = 10))[1]\n",
    "        print(label)\n",
    "        (x, y, w, h) = (x+region[0][0], y+region[0][1], w, h)\n",
    "        cv2.rectangle(orig_img, (x, y), (x + w, y + h), (0, 255, 0), 2)\n",
    "        cv2.putText(orig_img, label, (x , y + 90), cv2.FONT_HERSHEY_COMPLEX, 2, (255, 0, 0), 2)\n",
    "        cv2.imshow(\"image\", orig_img)\n",
    "        cv2.waitKey(0) \n",
    "        \n",
    "cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
